{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import albert\n",
    "import albert.modeling, albert.tokenization\n",
    "import transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, \"/home/husein/parsing/self-attentive-parser/src\")\n",
    "sys.path.append(\"/home/husein/parsing/self-attentive-parser\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import itertools\n",
    "import os.path\n",
    "import time\n",
    "import shutil\n",
    "import re\n",
    "import json\n",
    "\n",
    "import torch\n",
    "import torch.optim.lr_scheduler\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import evaluate\n",
    "import trees_newline as trees\n",
    "import vocabulary\n",
    "import nkutil\n",
    "import parse_nk_albert_tiny as parse_nk\n",
    "tokens = parse_nk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "if parse_nk.use_cuda:\n",
    "    info = torch.load('models/albert-tiny.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "parser = parse_nk.NKChartParser.from_spec(info['spec'], info['state_dict'])\n",
    "bert_model = info['spec']['hparams']['bert_model']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_treebank = trees.load_trees('test.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "tf.compat.v1.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "sd = parser.state_dict()\n",
    "\n",
    "LABEL_VOCAB = [x[0] for x in sorted(parser.label_vocab.indices.items(), key=lambda x: x[1])]\n",
    "TAG_VOCAB = [x[0] for x in sorted(parser.tag_vocab.indices.items(), key=lambda x: x[1])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "position_table = tf.constant(sd['embedding.position_table'].cpu().numpy(), name=\"position_table\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_layer_norm(input, torch_name, name):\n",
    "    # TODO(nikita): The epsilon here isn't quite the same as in pytorch\n",
    "    # The pytorch code adds eps=1e-3 to the standard deviation, while this\n",
    "    # tensorflow code adds eps=1e-6 to the variance.\n",
    "    # However, the resulting mismatch in floating-point values does not seem to\n",
    "    # translate to any noticable changes in the parser's tree output\n",
    "    mean, variance = tf.nn.moments(input, [1], keep_dims=True)\n",
    "    return tf.nn.batch_normalization(\n",
    "        input,\n",
    "        mean, variance,\n",
    "        offset=tf.constant(sd[f'{torch_name}.b_2'].cpu().numpy(), name=f\"{name}/offset\"),\n",
    "        scale=tf.constant(sd[f'{torch_name}.a_2'].cpu().numpy(), name=f\"{name}/scale\"),\n",
    "        variance_epsilon=1e-6)\n",
    "\n",
    "\n",
    "def make_heads(input, shape_bthf, shape_xtf, torch_name, name):\n",
    "    res = tf.matmul(input,\n",
    "        tf.constant(sd[torch_name].cpu().numpy().transpose((1,0,2)).reshape((512, -1)), name=f\"{name}/W\"))\n",
    "    res = tf.reshape(res, shape_bthf)\n",
    "    res = tf.transpose(res, (0,2,1,3)) # batch x num_heads x time x feat\n",
    "    res = tf.reshape(res, shape_xtf) # _ x time x feat\n",
    "    return res\n",
    "\n",
    "def make_attention(input, nonpad_ids, dim_flat, dim_padded, valid_mask, torch_name, name):\n",
    "    input_flat = tf.scatter_nd(indices=nonpad_ids[:, None], updates=input, shape=tf.concat([dim_flat, tf.shape(input)[1:]], axis=0))\n",
    "    input_flat_dat, input_flat_pos = tf.split(input_flat, 2, axis=-1)\n",
    "\n",
    "    shape_bthf = tf.concat([dim_padded, [8, -1]], axis=0)\n",
    "    shape_bhtf = tf.convert_to_tensor([dim_padded[0], 8, dim_padded[1], -1])\n",
    "    shape_xtf = tf.convert_to_tensor([dim_padded[0] * 8, dim_padded[1], -1])\n",
    "    shape_xf = tf.concat([dim_flat, [-1]], axis=0)\n",
    "\n",
    "    qs1 = make_heads(input_flat_dat, shape_bthf, shape_xtf, f'{torch_name}.w_qs1', f'{name}/q_dat')\n",
    "    ks1 = make_heads(input_flat_dat, shape_bthf, shape_xtf, f'{torch_name}.w_ks1', f'{name}/k_dat')\n",
    "    vs1 = make_heads(input_flat_dat, shape_bthf, shape_xtf, f'{torch_name}.w_vs1', f'{name}/v_dat')\n",
    "    qs2 = make_heads(input_flat_pos, shape_bthf, shape_xtf, f'{torch_name}.w_qs2', f'{name}/q_pos')\n",
    "    ks2 = make_heads(input_flat_pos, shape_bthf, shape_xtf, f'{torch_name}.w_ks2', f'{name}/k_pos')\n",
    "    vs2 = make_heads(input_flat_pos, shape_bthf, shape_xtf, f'{torch_name}.w_vs2', f'{name}/v_pos')\n",
    "\n",
    "    qs = tf.concat([qs1, qs2], axis=-1)\n",
    "    ks = tf.concat([ks1, ks2], axis=-1)\n",
    "    attn_logits = tf.matmul(qs, ks, transpose_b=True) / (1024 ** 0.5)\n",
    "\n",
    "    attn_mask = tf.reshape(tf.tile(valid_mask, [1,8*dim_padded[1]]), tf.shape(attn_logits))\n",
    "    # TODO(nikita): use tf.where and -float('inf') here?\n",
    "    attn_logits -= 1e10 * tf.to_float(~attn_mask)\n",
    "\n",
    "    attn = tf.nn.softmax(attn_logits)\n",
    "\n",
    "    attended_dat_raw = tf.matmul(attn, vs1)\n",
    "    attended_dat_flat = tf.reshape(tf.transpose(tf.reshape(attended_dat_raw, shape_bhtf), (0,2,1,3)), shape_xf)\n",
    "    attended_dat = tf.gather(attended_dat_flat, nonpad_ids)\n",
    "    attended_pos_raw = tf.matmul(attn, vs2)\n",
    "    attended_pos_flat = tf.reshape(tf.transpose(tf.reshape(attended_pos_raw, shape_bhtf), (0,2,1,3)), shape_xf)\n",
    "    attended_pos = tf.gather(attended_pos_flat, nonpad_ids)\n",
    "\n",
    "    out_dat = tf.matmul(attended_dat, tf.constant(sd[f'{torch_name}.proj1.weight'].cpu().numpy().transpose()))\n",
    "    out_pos = tf.matmul(attended_pos, tf.constant(sd[f'{torch_name}.proj2.weight'].cpu().numpy().transpose()))\n",
    "\n",
    "    out = tf.concat([out_dat, out_pos], -1)\n",
    "    return make_layer_norm(input + out, f'{torch_name}.layer_norm', f'{name}/layer_norm')\n",
    "\n",
    "def make_dense_relu_dense(input, torch_name, torch_type, name):\n",
    "    # TODO: use name\n",
    "    mul1 = tf.matmul(input, tf.constant(sd[f'{torch_name}.w_1{torch_type}.weight'].cpu().numpy().transpose()))\n",
    "    mul1b = tf.nn.bias_add(mul1, tf.constant(sd[f'{torch_name}.w_1{torch_type}.bias'].cpu().numpy()))\n",
    "    mul1b = tf.nn.relu(mul1b)\n",
    "    mul2 = tf.matmul(mul1b, tf.constant(sd[f'{torch_name}.w_2{torch_type}.weight'].cpu().numpy().transpose()))\n",
    "    mul2b = tf.nn.bias_add(mul2, tf.constant(sd[f'{torch_name}.w_2{torch_type}.bias'].cpu().numpy()))\n",
    "    return mul2b\n",
    "\n",
    "def make_ff(input, torch_name, name):\n",
    "    # TODO: use name\n",
    "    input_dat, input_pos = tf.split(input, 2, axis=-1)\n",
    "    out_dat = make_dense_relu_dense(input_dat, torch_name, 'c', name=\"TODO_dat\")\n",
    "    out_pos = make_dense_relu_dense(input_pos, torch_name, 'p', name=\"TODO_pos\")\n",
    "    out = tf.concat([out_dat, out_pos], -1)\n",
    "    return make_layer_norm(input + out, f'{torch_name}.layer_norm', f'{name}/layer_norm')\n",
    "\n",
    "def make_stacks(input, nonpad_ids, dim_flat, dim_padded, valid_mask, num_stacks):\n",
    "    res = input\n",
    "    for i in range(num_stacks):\n",
    "        res = make_attention(res, nonpad_ids, dim_flat, dim_padded, valid_mask, f'encoder.attn_{i}', name=f'attn_{i}')\n",
    "        res = make_ff(res, f'encoder.ff_{i}', name=f'ff_{i}')\n",
    "    return res\n",
    "\n",
    "def make_layer_norm_with_constants(input, constants):\n",
    "    # TODO(nikita): The epsilon here isn't quite the same as in pytorch\n",
    "    # The pytorch code adds eps=1e-3 to the standard deviation, while this\n",
    "    # tensorflow code adds eps=1e-6 to the variance.\n",
    "    # However, the resulting mismatch in floating-point values does not seem to\n",
    "    # translate to any noticable changes in the parser's tree output\n",
    "    mean, variance = tf.nn.moments(input, [1], keep_dims=True)\n",
    "    return tf.nn.batch_normalization(\n",
    "        input,\n",
    "        mean, variance,\n",
    "        offset=constants[0],\n",
    "        scale=constants[1],\n",
    "        variance_epsilon=1e-6)\n",
    "\n",
    "def make_flabel_with_constants(input, constants):\n",
    "    mul1 = tf.matmul(input, constants[0])\n",
    "    mul1b = tf.nn.bias_add(mul1, constants[1])\n",
    "    mul1b = make_layer_norm_with_constants(mul1b, constants[2:4])\n",
    "    mul1b = tf.nn.relu(mul1b)\n",
    "    mul2 = tf.matmul(mul1b, constants[4])\n",
    "    mul2b = tf.nn.bias_add(mul2, constants[5], name='flabel')\n",
    "    return mul2b\n",
    "\n",
    "def make_ftag(input):\n",
    "    constants = (\n",
    "        tf.constant(sd['f_tag.0.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_tag.0.bias'].cpu().numpy()),\n",
    "        tf.constant(sd['f_tag.1.b_2'].cpu().numpy(), name=\"tag/layer_norm/offset\"),\n",
    "        tf.constant(sd['f_tag.1.a_2'].cpu().numpy(), name=\"tag/layer_norm/scale\"),\n",
    "        tf.constant(sd['f_tag.3.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_tag.3.bias'].cpu().numpy()),\n",
    "    )\n",
    "    mul1 = tf.matmul(input, constants[0])\n",
    "    mul1b = tf.nn.bias_add(mul1, constants[1])\n",
    "    mul1b = make_layer_norm_with_constants(mul1b, constants[2:4])\n",
    "    mul1b = tf.nn.relu(mul1b)\n",
    "    mul2 = tf.matmul(mul1b, constants[4])\n",
    "    mul2b = tf.nn.bias_add(mul2, constants[5], name='ftag')\n",
    "    return mul2b\n",
    "\n",
    "def make_flabel_constants():\n",
    "    return (\n",
    "        tf.constant(sd['f_label.0.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_label.0.bias'].cpu().numpy()),\n",
    "        tf.constant(sd['f_label.1.b_2'].cpu().numpy(), name=\"label/layer_norm/offset\"),\n",
    "        tf.constant(sd['f_label.1.a_2'].cpu().numpy(), name=\"label/layer_norm/scale\"),\n",
    "        tf.constant(sd['f_label.3.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_label.3.bias'].cpu().numpy()),\n",
    "    )\n",
    "\n",
    "def make_network():\n",
    "    # batch x num_subwords\n",
    "    input_ids = tf.placeholder(shape=(None, None), dtype=tf.int32, name='input_ids')\n",
    "    word_end_mask = tf.placeholder(shape=(None, None), dtype=tf.int32, name='word_end_mask')\n",
    "    input_dat, nonpad_ids, dim_flat, dim_padded, valid_mask, lengths = make_bert(input_ids, word_end_mask)\n",
    "    input_pos_flat = tf.tile(position_table[:dim_padded[1]], [dim_padded[0], 1])\n",
    "    input_pos = tf.gather(input_pos_flat, nonpad_ids)\n",
    "\n",
    "    input_joint = tf.concat([input_dat, input_pos], -1)\n",
    "    input_joint = make_layer_norm(input_joint, 'embedding.layer_norm', 'embedding/layer_norm')\n",
    "\n",
    "    word_out = make_stacks(input_joint, nonpad_ids, dim_flat, dim_padded, valid_mask, num_stacks=parser.spec['hparams']['num_layers'])\n",
    "    word_out = tf.concat([word_out[:, 0::2], word_out[:, 1::2]], -1)\n",
    "\n",
    "    # part-of-speech predictions\n",
    "    ftag = make_ftag(word_out)\n",
    "    tags_packed = tf.argmax(ftag, axis=-1)\n",
    "    tags = tf.reshape(\n",
    "        tf.scatter_nd(indices=nonpad_ids[:, None], updates=tags_packed, shape=dim_flat),\n",
    "        dim_padded\n",
    "        )\n",
    "    tags = tf.identity(tags, name=\"tags\")\n",
    "\n",
    "    fp_out = tf.concat([word_out[:-1,:512], word_out[1:,512:]], -1)\n",
    "\n",
    "    fp_start_idxs = tf.cumsum(lengths, exclusive=True)\n",
    "    fp_end_idxs = tf.cumsum(lengths) - 1 # the number of fenceposts is 1 less than the number of words\n",
    "\n",
    "    fp_end_idxs_uneven = fp_end_idxs - tf.convert_to_tensor([1, 0])\n",
    "\n",
    "    # Have to make these outside tf.map_fn for model compression to work\n",
    "    constants = make_flabel_constants()\n",
    "\n",
    "    def to_map(start_and_end):\n",
    "        start, end = start_and_end\n",
    "        fp = fp_out[start:end]\n",
    "        flabel = make_flabel_with_constants(tf.reshape(fp[None,:,:] - fp[:,None,:], (-1, 1024)), constants)\n",
    "        actual_chart_size = end-start\n",
    "        flabel = tf.reshape(flabel, [actual_chart_size, actual_chart_size, -1])\n",
    "        amount_to_pad = dim_padded[1] - actual_chart_size\n",
    "        # extra padding on the label dimension is for the not-a-constituent label,\n",
    "        # which always has a score of 0\n",
    "        flabel = tf.pad(flabel, [[0, amount_to_pad], [0, amount_to_pad], [1, 0]])\n",
    "        return flabel\n",
    "\n",
    "    charts = tf.map_fn(to_map, (fp_start_idxs, fp_end_idxs), dtype=(tf.float32))\n",
    "    charts = tf.identity(charts, name=\"charts\")\n",
    "\n",
    "    return input_ids, word_end_mask, charts, tags\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_bert(input_ids, word_end_mask):\n",
    "    # We can derive input_mask from either input_ids or word_end_mask\n",
    "    input_mask = (1 - tf.cumprod(1 - word_end_mask, axis=-1, reverse=True))\n",
    "    token_type_ids = tf.zeros_like(input_ids)\n",
    "    bert_model = make_bert_instance(input_ids, input_mask, token_type_ids)\n",
    "\n",
    "    bert_features = bert_model.get_sequence_output()\n",
    "    bert_features_packed = tf.gather(\n",
    "        tf.reshape(bert_features, [-1, int(bert_features.shape[-1])]),\n",
    "        tf.to_int32(tf.where(tf.reshape(word_end_mask, (-1,))))[:,0])\n",
    "    projected_annotations = tf.matmul(\n",
    "        bert_features_packed,\n",
    "        tf.constant(sd['project_bert.weight'].cpu().numpy().transpose()))\n",
    "\n",
    "    # input_mask is over subwords, whereas valid_mask is over words\n",
    "    sentence_lengths = tf.reduce_sum(word_end_mask, -1)\n",
    "    valid_mask = (tf.range(tf.reduce_max(sentence_lengths))[None,:] < sentence_lengths[:, None])\n",
    "    dim_padded = tf.shape(valid_mask)[:2]\n",
    "    mask_flat = tf.reshape(valid_mask, (-1,))\n",
    "    dim_flat = tf.shape(mask_flat)[:1]\n",
    "    nonpad_ids = tf.to_int32(tf.where(mask_flat)[:,0])\n",
    "\n",
    "    return projected_annotations, nonpad_ids, dim_flat, dim_padded, valid_mask, sentence_lengths\n",
    "\n",
    "def make_bert_instance(input_ids, input_mask, token_type_ids):\n",
    "    # Transfer BERT config into tensorflow implementation\n",
    "    config = albert.modeling.AlbertConfig.from_dict(parser.bert.config.to_dict())\n",
    "    config.hidden_act = 'gelu'\n",
    "    model = albert.modeling.AlbertModel(config=config, is_training=False,\n",
    "        input_ids=input_ids, input_mask=input_mask, token_type_ids=token_type_ids)\n",
    "\n",
    "    # Next, transfer learned weights (after fine-tuning)\n",
    "    bert_variables = [v for v in tf.get_collection('variables') if 'bert' in v.name]\n",
    "    tf.variables_initializer(bert_variables).run()\n",
    "    for variable in bert_variables:\n",
    "        name = variable.name.split(':')[0]\n",
    "\n",
    "        name = name.replace(\"module/\", \"\")\n",
    "\n",
    "        # Renaming and simplifying\n",
    "        name = name.replace(\"ffn_1\", \"ffn\")\n",
    "        # name = name.replace(\"bert/\", \"albert/\")\n",
    "        name = name.replace(\"attention_1\", \"attention\")\n",
    "        name = name.replace(\"transform/\", \"\")\n",
    "        name = name.replace(\"LayerNorm_1\", \"full_layer_layer_norm\")\n",
    "        name = name.replace(\"LayerNorm\", \"attention/LayerNorm\")\n",
    "        name = name.replace(\"transformer/\", \"\")\n",
    "\n",
    "        # The feed forward layer had an 'intermediate' step which has been abstracted away\n",
    "        name = name.replace(\"intermediate/dense/\", \"\")\n",
    "        name = name.replace(\"ffn/intermediate/output/dense/\", \"ffn_output/\")\n",
    "\n",
    "        # ALBERT attention was split between self and output which have been abstracted away\n",
    "        name = name.replace(\"/output/\", \"/\")\n",
    "        name = name.replace(\"/self/\", \"/\")\n",
    "\n",
    "        # The pooler is a linear layer\n",
    "        name = name.replace(\"pooler/dense\", \"pooler\")\n",
    "\n",
    "        # The classifier was simplified to predictions from cls/predictions\n",
    "        name = name.replace(\"cls/predictions\", \"predictions\")\n",
    "        name = name.replace(\"predictions/attention\", \"predictions\")\n",
    "\n",
    "        # Naming was changed to be more explicit\n",
    "        name = name.replace(\"embeddings/attention\", \"embeddings\")\n",
    "        name = name.replace(\"inner_group_\", \"albert_layers/\")\n",
    "        name = name.replace(\"group_\", \"albert_layer_groups/\")\n",
    "\n",
    "        # Classifier\n",
    "        if len(name.split(\"/\")) == 1 and (\"output_bias\" in name or \"output_weights\" in name):\n",
    "            name = \"classifier/\" + name\n",
    "\n",
    "        # No ALBERT model currently handles the next sentence prediction task\n",
    "        if \"seq_relationship\" in name:\n",
    "            name = name.replace(\"seq_relationship/output_\", \"sop_classifier/classifier/\")\n",
    "            name = name.replace(\"weights\", \"weight\")\n",
    "\n",
    "        name = name.split('/')\n",
    "        array = variable.eval()\n",
    "        # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v\n",
    "        # which are not required for using pretrained model\n",
    "        if any(n in [\"adam_v\", \"adam_m\"] for n in name):\n",
    "            print(\"Skipping {}\".format(\"/\".join(name)))\n",
    "            continue\n",
    "        pytorch_var = parser\n",
    "        for m_name in name:\n",
    "\n",
    "            if re.fullmatch(r'[A-Za-z]+_\\d+', m_name):\n",
    "                l = re.split(r'_(\\d+)', m_name)\n",
    "            else:\n",
    "                l = [m_name]\n",
    "            if l[0] == 'kernel' or l[0] == 'gamma':\n",
    "                pytorch_var = getattr(pytorch_var, 'weight')\n",
    "            elif l[0] == 'output_bias' or l[0] == 'beta':\n",
    "                pytorch_var = getattr(pytorch_var, 'bias')\n",
    "            elif l[0] == 'output_weights':\n",
    "                pytorch_var = getattr(pytorch_var, 'weight')\n",
    "            elif l[0] == 'cls':\n",
    "                pytorch_var = getattr(pytorch_var, 'cls')\n",
    "            else:\n",
    "                pytorch_var = getattr(pytorch_var, l[0])\n",
    "            if len(l) >= 2:\n",
    "                try:\n",
    "                    num = int(l[1])\n",
    "                    pytorch_var = pytorch_var[num]\n",
    "                except Exception as e:\n",
    "                    print(pytorch_var)\n",
    "                    raise\n",
    "        if m_name[-11:] == '_embeddings':\n",
    "            pytorch_var = getattr(pytorch_var, 'weight')\n",
    "        elif m_name == 'kernel':\n",
    "            pytorch_var = pytorch_var.t()\n",
    "        try:\n",
    "            assert pytorch_var.shape == array.shape\n",
    "        except AssertionError as e:\n",
    "            print(e)\n",
    "            e.args += (pytorch_var.shape, array.shape)\n",
    "            raise\n",
    "\n",
    "        variable.load(pytorch_var.detach().cpu().numpy())\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/modeling.py:194: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/modeling.py:507: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/modeling.py:588: The name tf.assert_less_equal is deprecated. Please use tf.compat.v1.assert_less_equal instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/modeling.py:1025: The name tf.AUTO_REUSE is deprecated. Please use tf.compat.v1.AUTO_REUSE instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/modeling.py:253: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.Dense instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/layers/core.py:187: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.__call__` method instead.\n",
      "WARNING:tensorflow:From <ipython-input-11-03800aaca47e>:120: Variable.load (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Prefer Variable.assign which has equivalent behavior in 2.X.\n",
      "WARNING:tensorflow:From <ipython-input-11-03800aaca47e>:10: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From <ipython-input-11-03800aaca47e>:10: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n",
      "WARNING:tensorflow:From <ipython-input-10-dc457fb7a117>:46: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    }
   ],
   "source": [
    "the_inp_tokens, the_inp_mask, the_out_chart, the_out_tags = make_network()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from parse_nk import BERT_TOKEN_MAPPING\n",
    "def bertify_batch(sentences):\n",
    "    all_input_ids = np.zeros((len(sentences), parser.bert_max_len), dtype=int)\n",
    "    all_word_end_mask = np.zeros((len(sentences), parser.bert_max_len), dtype=int)\n",
    "\n",
    "    subword_max_len = 0\n",
    "    for snum, sentence in enumerate(sentences):\n",
    "        tokens = []\n",
    "        word_end_mask = []\n",
    "\n",
    "        tokens.append(\"[CLS]\")\n",
    "        word_end_mask.append(1)\n",
    "\n",
    "        cleaned_words = []\n",
    "        for word in sentence:\n",
    "            word = BERT_TOKEN_MAPPING.get(word, word)\n",
    "            if word == \"n't\" and cleaned_words:\n",
    "                cleaned_words[-1] = cleaned_words[-1] + \"n\"\n",
    "                word = \"'t\"\n",
    "            cleaned_words.append(word)\n",
    "\n",
    "        for word in cleaned_words:\n",
    "            word_tokens = parser.bert_tokenizer.tokenize(word)\n",
    "            for _ in range(len(word_tokens)):\n",
    "                word_end_mask.append(0)\n",
    "            word_end_mask[-1] = 1\n",
    "            tokens.extend(word_tokens)\n",
    "        tokens.append(\"[SEP]\")\n",
    "        word_end_mask.append(1)\n",
    "\n",
    "        input_ids = parser.bert_tokenizer.convert_tokens_to_ids(tokens)\n",
    "\n",
    "        subword_max_len = max(subword_max_len, len(input_ids))\n",
    "\n",
    "        all_input_ids[snum, :len(input_ids)] = input_ids\n",
    "        all_word_end_mask[snum, :len(word_end_mask)] = word_end_mask\n",
    "\n",
    "    all_input_ids = all_input_ids[:, :subword_max_len]\n",
    "    all_word_end_mask = all_word_end_mask[:, :subword_max_len]\n",
    "    return all_input_ids, all_word_end_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 28/28 [00:01<00:00, 14.26it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "eval_batch_size = 16\n",
    "test_predicted = []\n",
    "for start_index in tqdm(range(0, len(test_treebank), eval_batch_size)):\n",
    "    subbatch_trees = test_treebank[start_index:start_index+eval_batch_size]\n",
    "    subbatch_sentences = [[leaf.word for leaf in tree.leaves()] for tree in subbatch_trees]\n",
    "    inp_val_tokens, inp_val_mask = bertify_batch([[word for word in sentence] for sentence in subbatch_sentences])\n",
    "    out_val_chart, out_val_tags = sess.run((the_out_chart, the_out_tags), \n",
    "                                       {the_inp_tokens: inp_val_tokens, the_inp_mask: inp_val_mask})\n",
    "    trees = []\n",
    "    scores = []\n",
    "    for snum, sentence in enumerate(subbatch_sentences):\n",
    "        chart_size = len(sentence) + 1\n",
    "        tf_chart = out_val_chart[snum,:chart_size,:chart_size,:]\n",
    "        sentence = list(zip([TAG_VOCAB[idx] for idx in out_val_tags[snum,1:chart_size]], [x for x in sentence]))\n",
    "        tree, score = parser.decode_from_chart(sentence, tf_chart)\n",
    "        trees.append(tree)\n",
    "        scores.append(score)\n",
    "    test_predicted.extend([p.convert() for p in trees])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<trees_newline.InternalTreebankNode object at 0x7f231169e128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc296e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231056ce10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2999e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231056e710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc29c438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231056ffd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc29ce48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23105738d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc29e908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23105771d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2a13c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310578a90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2a1d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231057d438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2a56d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231057ef28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2a72b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310581b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2a7e10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310585780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2aab38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231058b390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2ae780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231058def0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2b0438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310591b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2b0f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310594780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2b3c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23105991d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2b7940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231059add8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc292400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231059df98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2964e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23105a1c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc296b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23105a4710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc239c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23105a5f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc28d198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310527e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc28dc50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310529d68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc23e358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231052bc88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc23ea58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231052cba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc241160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231052ea90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc241828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310531978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc241e48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310532898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc243518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23105347b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc243ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310536c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc246278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23105387b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc246c18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231053a2b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2490f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231053ad68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2922e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231053c898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2498d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231053e390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2499b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231053eeb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc249e48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310541978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc28f390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310543470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc28f780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310547518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc28fba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231054b550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc24d7b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231054e668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc24f4a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310551828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc251128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23105529e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc251898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310556be0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc251f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310558e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc254748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231055d080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc254eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231055e400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2577b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23105607b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc25a048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310561be0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc260be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310565cc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2622e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104eb978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2650b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104ed9e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2683c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104eea90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc268a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104f1b38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc26b0b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104f2c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc26b748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104f4c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc26bda0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104f7cc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc26e470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104f9da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc26eb00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104fbe80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc271198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104ffa90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2717f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23105026d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc274438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310502e80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc277080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310508048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc277390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310508630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1fa240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310508c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc282e10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231050b240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2825f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231050b7f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc282c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231050bdd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc282ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231050c470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2869b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231050ca90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc286080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231050e0b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc286320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23105117b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc286a20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310515ba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc290198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23105176d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc25afd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310519240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc260278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310519d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2607b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231051b860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1fa908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231051d390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1fac50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231051de80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1fb0f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23105209b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1fb518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23105214e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc20d898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310522be0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc20dd30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23105254e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2107b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310525da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc210b38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104a7710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc210ef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104a9080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc214278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104a9940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc214630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104ab2b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2149b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104abbe0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc214d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104ad550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2160f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104aee48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc216470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104b4668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc216ef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104b7e80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc21b0f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104bf5c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc21f128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104c3da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc223278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104c95f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc224390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104ceda0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1bc0b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104d1630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1be2b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104d5d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1becf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104d8438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1c1780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104dbc18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1c4128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104de390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1c4b38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104e0ba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1c85f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104e5320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1c8fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104e6a90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1cab00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231046b2e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1cd550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231046fda0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1cdf28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310476860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1d23c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231047d390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1d6828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310480da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1d9d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104878d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1e0320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231048d400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1e2828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231048f2e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc22a240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310492198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc22aba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310494080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc230668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310494f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2307f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310495e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc231978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231049acf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc231320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231049bbe0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc231c18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231049dac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc27c0f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104a0438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc27cc50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104a3d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc233518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310427668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc236748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310429ef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1e5f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231042e898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc22b940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310431208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1ea320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310433b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1eacf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310438518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1f0748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310438b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2f45f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231043b208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2f4860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231043b860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2f48d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231043be80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2f4d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231043c4e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2f4b70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231043cb00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2f4ef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231043f198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc228748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231043f7f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc228c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104429b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc2287f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310446550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc27a710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310448c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1f2860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310451390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1791d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104552e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc17c908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310455828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc17f4a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310455d68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc17f6a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104592e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc17f8d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104597f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc17fcf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310459cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc17fe80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231045a278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1860b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231045a780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc186278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231045acc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc186400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310461780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1865c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104636d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc189908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104656a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc189f60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103e7588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc18b5c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103e94a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc18bbe0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103ec438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc18d278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103ee390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc18d898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103ef2b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc18deb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103f2208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc191550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103f3a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc191b70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103f83c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc194588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103fad68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1aa470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103fe898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1aada0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310402240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1ac748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310404c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1b00b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310409710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1b0a20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231040c0b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1b3438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104105c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1b3d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310413f60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1b6e10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310417908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1399e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231041c320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc13c550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231041ecf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc13e128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104216d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc13ecc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23104260f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc142898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103a8a90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc145470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103ac4a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc147048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103af9e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc147be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103b4eb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc150080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103bb4a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc155198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103c09e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc156320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103c3ef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc15a4a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103ca518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc15d6d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103ceac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc160898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103d50f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc163a20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103d66a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc165be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103d8cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1694e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103dd320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc169e48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103de898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc16c7f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103e1ef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc16e1d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103e4518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc16eb38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310367ac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1724e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231036b128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc172dd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231036c7b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc176748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103705c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1095f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310372390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc109be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310374198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc10c1d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310374f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc10c748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310376d68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc10ccc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310378ba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc10f2b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103799e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc10f860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310383128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc10fe10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310388860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc116588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231038ef28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc118da0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103976a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc120588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231039ce80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc122d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103a6550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc128588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231032ac88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc12ccf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310334400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1315f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231033b048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc135e10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231033ec18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1a72b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103447f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc14d240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231034b3c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0ba400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231034ef98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc105390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310355b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0be828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231035c748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0c0b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310362390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0c2eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310362e80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0c7240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23103659e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0c7668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102e7550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0c7a20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102e90b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0c7e10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102e9c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0ca278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102eb780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0ca668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102ee2b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0caa58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102eee10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0cadd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102f1a90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0ce208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102f56d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0e7080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102fa358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0e7dd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102fcfd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0eab00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310300c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0ec748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310304860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0ee438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310308588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0f41d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231030e1d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0f4f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231030fe48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0f5b70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310312d68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1a0860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310317c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc07a5f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310321550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc07f2e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310325cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc081b38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102b0588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0872e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102b6d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc08ab38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102c0550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc090400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102c7d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc093c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102d0518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0da470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102d5cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0ddc18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102da550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc099e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102dde80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0d2978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102e4828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0a1cc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231026a1d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0a2f60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231026fa20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0a9320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310275358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0ad630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310279cc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0b08d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231027d080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0b4c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310280400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0b55f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310281748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0b5f60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310284b38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc039940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310285cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc03c208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310287ef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc03ca58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231028c080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc040160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231028e208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc04d748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102903c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc04ddd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310292588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc050588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231029a6a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc050dd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102a0c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc055780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310229240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc05a048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231022f780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc05d8d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310235cc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc063160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231023e390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc066908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310245a20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc06c208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231024b0b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc06fac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310252518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc072be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310256550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc078358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310257da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87b2208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231025c6a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87b2cc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231025ef28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87b58d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310261828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1a4b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102660b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1a4b38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101e8908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1a7630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101ec208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0fa2e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101eea90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0fad30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101f1f60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0d73c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101f5550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc040ef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101f6a20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87b85c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101f9ef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87b8f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101fc438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc047978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310200908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc044320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310202da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc044cc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310206320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc041748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23102092e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87be0f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231020e2b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87befd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310213278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87c1e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310217278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87cc470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231021c240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87cf128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231021f208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87cfe10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310224128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87d3c18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101a8128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87d69e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101a9f60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87d98d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101ae550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87dc5c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101b07f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87dcfd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101b25c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87df860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101b7ef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87e2588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101babe0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87e2ef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101be908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87e5a90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101c45f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87e8668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101c72b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87eb198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101c8f98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87ebd30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101cdcc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87ef898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101d24e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc0dada0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101d4cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8771908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101d9588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8776400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101dad30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8776e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101e0550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87789e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101e1d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b877a518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101e65f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b877d0b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310168dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b877dbe0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231016d358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8780780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231016f898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8783160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310171e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8783b38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310175470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8786518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310177978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8786eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231017aef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8789940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231017f550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b878c358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310180b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b878ccf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310184e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8797a90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231018b0f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87969b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231018e390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8798898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101935c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b879f6d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101987f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87a0588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310199940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87a3550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101a0748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87a3cf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101a1c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87a9160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101a6198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87a99e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310128630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87ac2e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231012aac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87acb70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231012df60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87af550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101304a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87afeb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310131a20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87337f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310135f98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8736128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231013b128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87369e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231013e2e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b878ea90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310143400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b873f390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310147518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8740278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231014a630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8743160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101506d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8743fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101537f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8747eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101579b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8749dd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310159d68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b874bd30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231015e160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b874e6a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310160518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b874eeb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310162898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87527b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310164c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87540b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310166fd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8754940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100eb390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8757160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100ec710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8757a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100f0160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b875a3c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100f4278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87704a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100f84a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86f3470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100fd748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86f5400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23101009e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86f8470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310104cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86fb5c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310109e80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86fe6a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310110160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8701630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310114470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87045f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310116b38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87066d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231011d6d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8709748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310121d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b870dac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100a8438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8710d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100aba90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8713e48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100b21d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8719048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100b6828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b871d128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100b9ef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b871f208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100bf390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc19e470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100c07f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc1a2ac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100c3c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b875d3c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100c80b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b875dba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100cb4e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8765208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100cc940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8765b38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100cecf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8767780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100d3198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8767c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100d45f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b876a7b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100d9128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b876aef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100dd9e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b876ecf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100e32b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8722dd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310068828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b872bd68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100699e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b872ad30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231006e588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b872d4e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310070a90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b872df60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100732b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc19ac88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310073a90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc19a400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100752b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc19a080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310075a90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc19ef60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100762b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc19e6d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310076a20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc19e4e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310078240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc19e908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310078a20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87cc278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100814e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8730a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100840b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86b92b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310088470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86b9e48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231008b080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86bc710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231008ccc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86be4a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100918d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86c1208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310096550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86c1f60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100991d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86c5da0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231009cd68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86d6518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100a09e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86db390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100a5668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86dd0f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100287f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86dde10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231002c860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86e0c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100319b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86e2b70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310035a20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86e59e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231003aa58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86e8780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231003eb70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86ec5f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310042cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86ed400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310045f98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8673240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100483c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8673c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231004b828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86766d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231004db70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b867a0b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231004feb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b867aac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100542e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b867d630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310056668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86c7fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100589e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86cee10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f231005f630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86d3fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f2310061cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b867fa20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f23100654e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86cf5c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f22fc7b9c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86810f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f22fc7bd400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8681ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f22fc7bfba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8685668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f22fc7c33c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8689160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f22fc7c4ba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8689be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f22fc7c93c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b868c6d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f22fc7c9f60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b868e1d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f22fc7ccb00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b868e710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f22fc7cf6a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b868ec18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f22fc7d0240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b8691198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f22fc7d0dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b86915f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f22fc7d19b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc19ab38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f22fc7d3518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22fc19a5c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f22fc7d50f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f22b87975f8>\n"
     ]
    }
   ],
   "source": [
    "test_fscore = evaluate.evalb('EVALB/', test_treebank[:len(test_predicted)], test_predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'(Recall=67.21, Precision=74.89, FScore=70.84, CompleteMatch=2.11, TaggingAccuracy=87.75)'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(test_fscore)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_node_names = [the_inp_tokens.name.split(':')[0], the_inp_mask.name.split(':')[0]]\n",
    "output_node_names = [the_out_chart.name.split(':')[0], the_out_tags.name.split(':')[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-18-f034bd79213f>:1: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n",
      "INFO:tensorflow:Froze 23 variables.\n",
      "INFO:tensorflow:Converted 23 variables to const ops.\n"
     ]
    }
   ],
   "source": [
    "graph_def = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, output_node_names)\n",
    "from tensorflow.tools.graph_transforms import TransformGraph\n",
    "graph_def = TransformGraph(graph_def, input_node_names, output_node_names, [\n",
    "'strip_unused_nodes()',\n",
    "'remove_nodes(op=Identity, op=CheckNumerics)',\n",
    "'fold_constants()',\n",
    "'fold_old_batch_norms',\n",
    "'fold_batch_norms',\n",
    "'round_weights(num_steps=128)',\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('export/albert-tiny.pb', 'wb') as f:\n",
    "    f.write(graph_def.SerializeToString())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('vocab-albert-tiny.json', 'w') as fopen:\n",
    "    json.dump({'label': LABEL_VOCAB, 'tag': TAG_VOCAB}, fopen)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
